#!/usr/bin/env python
# -*- coding: utf-8 -*-
# @Time : 2018-08-21 18:04:39
# @Author : By Joker
# @File : getBook2.py 
# @Software: PyCharm

from urllib import request

from bs4 import BeautifulSoup

'''
获取指定url的全部章节记忆正文,并写入txt
'''


# 获取指定url中的正文
def download_specified_chapter(txt_url, chapter_url, header, coding, chapter_name):
    # 生成request对象
    dowload_req = request.Request(chapter_url, headers=header)
    # 打开request对象中的url网址,并获得对应内容
    response = request.urlopen(dowload_req)
    # 解码之后,获取页面的html
    download_html = response.read().decode(coding, 'ignore')
    # 获取html的beautifulSoup
    origin_soup = BeautifulSoup(download_html, 'lxml')
    # 获取正文部分
    content = origin_soup.find(class_='content')
    # 整理正文格式
    txt = content.text.replace('\n' * 8, '\n').replace('nrgg360();', '')
    print("正在下载:{}----链接:{}".format(chapter_name, chapter_url))
    with open(txt_url, 'a') as f:
        if chapter_name is None:
            f.write('\n')
        else:
            f.write('\n\n' + chapter_name + '\n')
        f.write(txt)


# 获取指定url中的目录列表
def download_all_url(txt_url,main_url, target_url, header, coding):
    url_req = request.Request(target_url, headers=header)
    response = request.urlopen(url_req)
    html = response.read().decode(coding, "ignore")
    html_soup = BeautifulSoup(html, "lxml")
    # 整理正文
    for element in html_soup.find_all("li", class_="line2"):
        chapter_name = element.string
        chapter_url = main_url + element.a.get('href')
        download_specified_chapter(txt_url,chapter_url, header, coding, chapter_name)


if __name__ == "__main__":
    main_url = "http://www.75xs.cc"
    target_url = 'http://www.75xs.cc/book/longzu1/'
    txt_url = "E:/books/long/1/龙族1.txt"
    header = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.79 Safari/537.36 Maxthon/5.2.3.4000'}
    download_all_url(txt_url,main_url, target_url, header, 'utf-8')
